package org.nlp.algo.feature.ig;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.commons.io.FileUtils;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;
import org.nlp.lucene.BloomAnalyzer;

public class FeatureMI {

	/**
	 * @author huili
	 * 利用互信息算法实现特征提取
	 */
	
	// 训练语料文件夹
	private static String corpusPath = "resource/corpus/train";
	private static File TDIR = new File(corpusPath);
	
	//提取的特征
	public static String featurePatch="resource/corpus/feature/";
	
	//HashMap<String,HashMap<String,Set<String>>> 对应的是类别，文件名，文件中各词项
	public static HashMap<String,HashMap<String,Set<String>>> shl=new HashMap<String,HashMap<String,Set<String>>>();
	public static Set<String> classSet=new HashSet<String>();
	public static Set<String> wordsSet=new HashSet<String>();
		

	
	public static void main(String[] args) throws IOException 
	{		
		File[] tranTypes = TDIR.listFiles();
		for (int i = 0; i < tranTypes.length; i++) 
		{
			if (tranTypes[i].isDirectory()
					&& !tranTypes[i].getName().startsWith("."))
			{
				System.out.println(tranTypes[i]);
				comOneType(tranTypes[i]);
				
				
			}
		}

	/*	File f=new File(mypath+"shl.txt");
		BufferedWriter b=new BufferedWriter(new FileWriter(f));
		b.write(shl.toString());
		b.close();*/
		
		Iterator<String> itc=classSet.iterator();
		while(itc.hasNext())
		{
			String typeName=itc.next();

			Iterator<String> itw=wordsSet.iterator();
			HashMap<String,Double> pair=new HashMap<String,Double>();
			//System.out.println(typeName);
			while(itw.hasNext())
			{
				String wordName=itw.next();
				double igV=GetIG(typeName,wordName);
				pair.put(wordName, igV);
				//System.out.print(wordName+"\t"+igV+"\r\n");
			}
			
			// 排序 
			List<Map.Entry<String, Double>> info=SortPair2(pair);
			File fw=new File(featurePatch+typeName+".txt");
			if(fw.exists())
				fw.delete();
			else
				fw.createNewFile();
			BufferedWriter bw=new BufferedWriter(new FileWriter(fw));
			for (int i = 0; i < info.size(); i++) { 
				String id = info.get(i).toString(); 
				bw.write(id+"\r\n");
				}
			bw.close();
		}
		System.out.println();
	}
	
	//读取文件并分词，填加到多对多的集合中
	private static void comOneType(File file) throws IOException 
	{		
		String type = file.getName();
		File[] models = file.listFiles(new FileFilter() {
			@Override
			public boolean accept(File pathname) {				
				if (pathname.getName().endsWith(".txt")) {
					return true;
				}
				return false;
			}
		});		
		
		BloomAnalyzer analyzer = new BloomAnalyzer(Version.LUCENE_40); //直接用bloomAnalyzer类
		HashMap<String,Set<String>> hsl=new HashMap<String,Set<String>>();
		hsl.clear();
		//Analyzer analyzer=new IKAnalyzer();
		for (int e = 0; e < models.length; e++) {
			String str = FileUtils.readFileToString(models[e],"GBK");
//			FeatureAnalyzer analy = new FeatureAnalyzer();
			
			TokenStream token = analyzer.tokenStream("", new StringReader(
					str));
			CharTermAttribute term = token
					.addAttribute(CharTermAttribute.class);
			Set<String> wordList=new HashSet<String>();
			wordList.clear();
			while (token.incrementToken()) {
				String word = term.toString();			
				//System.out.println(word);
				wordList.add(word);
				wordsSet.add(word);
			}
			hsl.put(models[e].getName(),wordList);
		}
		shl.put(file.getName(),hsl);
		classSet.add(file.getName());
		analyzer.close();
		
	}
	//统计IG值中的各值 ，并返回
    public static double GetIG(String classType,String word)
    {    
    	int n00=0;
    	int n01=0;
    	int n10=0;
    	int n11=0;
    	//HashMap<String,HashMap<String,List<String>>>
    	Iterator<Entry<String,HashMap<String,Set<String>>>> itC=shl.entrySet().iterator();
    	while(itC.hasNext())
    	{
    		Entry<String,HashMap<String,Set<String>>> enC=itC.next();
    		
    		Iterator<Entry<String,Set<String>>> itD=enC.getValue().entrySet().iterator();
    		while(itD.hasNext())
    		{
    			Entry<String,Set<String>> enD=itD.next();
    			 
	    		if(!(enC.getKey().equalsIgnoreCase(classType)))//不属于该类
	    		{
	    			if(enD.getValue().contains(word))//包含
	    			{
	    				n10++;
	    			}
	    			else
	    			{
	    				n00++;
	    			}
	    		}//属于该类
	    		else
	    		{
	    			if(enD.getValue().contains(word))//包含
	    			{
	    				n11++;
	    			}
	    			else
	    			{
	    				n01++;
	    			}
	    		}
    		}
    	}
    	FeatureMI fi=new FeatureMI();
    	double igValue=fi.CalInformationGain((double)n11,(double)n10,(double)n01,(double)n00);
    	return igValue;
    }
    
    //排序
	public static List<Map.Entry<String, Double>> SortPair2(HashMap<String,Double> listPair)
	{
		List<Map.Entry<String, Double>> infoIds = new ArrayList<Map.Entry<String, Double>>( listPair.entrySet()); 
		Collections.sort(infoIds, new Comparator<Map.Entry<String, Double>>() { 
			public int compare(Map.Entry<String,Double> o1,Map.Entry<String,Double> o2)
			{
			if (o2.getValue() > o1.getValue())
			{
			return 1;
			}
			else if (o1.getValue() > o2.getValue())
			{
			return -1;
			}
			else
			{
			return 0;
			}
			}
			}); 
		return infoIds;
	}
	
	/************************************************************************/
	/* 计算IG值                                                                     */
	/************************************************************************/
	double CalInformationGain(double N11,double N10,double N01,double N00)
	{
	    double IG=0;
	    double N=N11+N10+N01+N00;
	    double N1=N10+N11;
	    double N0=N01+N00;
	    if(N*N11>0)
	    	IG+=(N11/N)*(Math.log((N*N11)/(N1*N1))/Math.log(2));
	    if(N*N01>0)
		    IG+=(N01/N)*(Math.log((N*N01)/(N0*N1))/Math.log(2));
	    if(N*N10>0)
		    IG+=(N10/N)*(Math.log((N*N10)/(N0*N1))/Math.log(2));
	    if(N*N00>0)
		    IG+=(N00/N)*(Math.log((N*N00)/(N0*N0))/Math.log(2));
	    
	    return IG;
	}
}

abstract class MyComparator implements Comparator{


public int compare(double o1, double o2) {
	// TODO Auto-generated method stub
	if(o1>o2) return 1;
	else if(o1==o2) return 0;
	else return -1;
}
}
